{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8. Convolutional Neural Networks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I recommend you take a look at these material first."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* http://web.stanford.edu/class/cs224n/lectures/cs224n-2017-lecture13-CNNs.pdf\n",
    "* http://www.aclweb.org/anthology/D14-1181\n",
    "* https://github.com/Shawn1993/cnn-text-classification-pytorch\n",
    "* http://cogcomp.org/Data/QA/QC/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import nltk\n",
    "import random\n",
    "import numpy as np\n",
    "from collections import Counter, OrderedDict\n",
    "import nltk\n",
    "import re\n",
    "from copy import deepcopy\n",
    "flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "random.seed(1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "USE_CUDA = torch.cuda.is_available()\n",
    "gpus = [0]\n",
    "torch.cuda.set_device(gpus[0])\n",
    "\n",
    "FloatTensor = torch.cuda.FloatTensor if USE_CUDA else torch.FloatTensor\n",
    "LongTensor = torch.cuda.LongTensor if USE_CUDA else torch.LongTensor\n",
    "ByteTensor = torch.cuda.ByteTensor if USE_CUDA else torch.ByteTensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getBatch(batch_size, train_data):\n",
    "    random.shuffle(train_data)\n",
    "    sindex = 0\n",
    "    eindex = batch_size\n",
    "    while eindex < len(train_data):\n",
    "        batch = train_data[sindex: eindex]\n",
    "        temp = eindex\n",
    "        eindex = eindex + batch_size\n",
    "        sindex = temp\n",
    "        yield batch\n",
    "    \n",
    "    if eindex >= len(train_data):\n",
    "        batch = train_data[sindex:]\n",
    "        yield batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pad_to_batch(batch):\n",
    "    x,y = zip(*batch)\n",
    "    max_x = max([s.size(1) for s in x])\n",
    "    x_p = []\n",
    "    for i in range(len(batch)):\n",
    "        if x[i].size(1) < max_x:\n",
    "            x_p.append(torch.cat([x[i], Variable(LongTensor([word2index['<PAD>']] * (max_x - x[i].size(1)))).view(1, -1)], 1))\n",
    "        else:\n",
    "            x_p.append(x[i])\n",
    "    return torch.cat(x_p), torch.cat(y).view(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def prepare_sequence(seq, to_index):\n",
    "    idxs = list(map(lambda w: to_index[w] if to_index.get(w) is not None else to_index[\"<UNK>\"], seq))\n",
    "    return Variable(LongTensor(idxs))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data load & Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TREC question dataset(http://cogcomp.org/Data/QA/QC/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Task involves\n",
    "classifying a question into 6 question\n",
    "types (whether the question is about person,\n",
    "location, numeric information, etc.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = open('../dataset/train_5500.label.txt', 'r', encoding='latin-1').readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = [[d.split(':')[1][:-1], d.split(':')[0]] for d in data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X, y = list(zip(*data))\n",
    "X = list(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Num masking "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It reduces the search space. ex. my birthday is 12.22 ==> my birthday is ##.##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i, x in enumerate(X):\n",
    "    X[i] = re.sub('\\d', '#', x).split()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build Vocab "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab = list(set(flatten(X)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9117"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(y)) # num of class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word2index={'<PAD>': 0, '<UNK>': 1}\n",
    "\n",
    "for vo in vocab:\n",
    "    if word2index.get(vo) is None:\n",
    "        word2index[vo] = len(word2index)\n",
    "        \n",
    "index2word = {v:k for k, v in word2index.items()}\n",
    "\n",
    "target2index = {}\n",
    "\n",
    "for cl in set(y):\n",
    "    if target2index.get(cl) is None:\n",
    "        target2index[cl] = len(target2index)\n",
    "\n",
    "index2target = {v:k for k, v in target2index.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_p, y_p = [], []\n",
    "for pair in zip(X,y):\n",
    "    X_p.append(prepare_sequence(pair[0], word2index).view(1, -1))\n",
    "    y_p.append(Variable(LongTensor([target2index[pair[1]]])).view(1, -1))\n",
    "    \n",
    "data_p = list(zip(X_p, y_p))\n",
    "random.shuffle(data_p)\n",
    "\n",
    "train_data = data_p[: int(len(data_p) * 0.9)]\n",
    "test_data = data_p[int(len(data_p) * 0.9):]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Pretrained word vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "you can download pretrained word vector from here https://github.com/mmihaltz/word2vec-GoogleNews-vectors "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = gensim.models.KeyedVectors.load_word2vec_format('../dataset/GoogleNews-vectors-negative300.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3000000"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model.index2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pretrained = []\n",
    "\n",
    "for key in word2index.keys():\n",
    "    try:\n",
    "        pretrained.append(model[word2index[key]])\n",
    "    except:\n",
    "        pretrained.append(np.random.randn(300))\n",
    "        \n",
    "pretrained_vectors = np.vstack(pretrained)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modeling "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"../images/08.cnn-for-text-architecture.png\">\n",
    "<center>borrowed image from http://www.aclweb.org/anthology/D14-1181</center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class  CNNClassifier(nn.Module):\n",
    "    \n",
    "    def __init__(self, vocab_size, embedding_dim, output_size, kernel_dim=100, kernel_sizes=(3, 4, 5), dropout=0.5):\n",
    "        super(CNNClassifier,self).__init__()\n",
    "\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.convs = nn.ModuleList([nn.Conv2d(1, kernel_dim, (K, embedding_dim)) for K in kernel_sizes])\n",
    "\n",
    "        # kernal_size = (K,D) \n",
    "        self.dropout = nn.Dropout(dropout)\n",
    "        self.fc = nn.Linear(len(kernel_sizes) * kernel_dim, output_size)\n",
    "    \n",
    "    \n",
    "    def init_weights(self, pretrained_word_vectors, is_static=False):\n",
    "        self.embedding.weight = nn.Parameter(torch.from_numpy(pretrained_word_vectors).float())\n",
    "        if is_static:\n",
    "            self.embedding.weight.requires_grad = False\n",
    "\n",
    "\n",
    "    def forward(self, inputs, is_training=False):\n",
    "        inputs = self.embedding(inputs).unsqueeze(1) # (B,1,T,D)\n",
    "        inputs = [F.relu(conv(inputs)).squeeze(3) for conv in self.convs] #[(N,Co,W), ...]*len(Ks)\n",
    "        inputs = [F.max_pool1d(i, i.size(2)).squeeze(2) for i in inputs] #[(N,Co), ...]*len(Ks)\n",
    "\n",
    "        concated = torch.cat(inputs, 1)\n",
    "\n",
    "        if is_training:\n",
    "            concated = self.dropout(concated) # (N,len(Ks)*Co)\n",
    "        out = self.fc(concated) \n",
    "        return F.log_softmax(out,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It takes for a while if you use just cpu."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "EPOCH = 5\n",
    "BATCH_SIZE = 50\n",
    "KERNEL_SIZES = [3,4,5]\n",
    "KERNEL_DIM = 100\n",
    "LR = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = CNNClassifier(len(word2index), 300, len(target2index), KERNEL_DIM, KERNEL_SIZES)\n",
    "model.init_weights(pretrained_vectors) # initialize embedding matrix using pretrained vectors\n",
    "\n",
    "if USE_CUDA:\n",
    "    model = model.cuda()\n",
    "    \n",
    "loss_function = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(model.parameters(), lr=LR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0/5] mean_loss : 2.13\n",
      "[1/5] mean_loss : 0.12\n",
      "[2/5] mean_loss : 0.08\n",
      "[3/5] mean_loss : 0.02\n",
      "[4/5] mean_loss : 0.05\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCH):\n",
    "    losses = []\n",
    "    for i,batch in enumerate(getBatch(BATCH_SIZE, train_data)):\n",
    "        inputs,targets = pad_to_batch(batch)\n",
    "        \n",
    "        model.zero_grad()\n",
    "        preds = model(inputs, True)\n",
    "        \n",
    "        loss = loss_function(preds, targets)\n",
    "        losses.append(loss.data.tolist()[0])\n",
    "        loss.backward()\n",
    "        \n",
    "        #for param in model.parameters():\n",
    "        #    param.grad.data.clamp_(-3, 3)\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        if i % 100 == 0:\n",
    "            print(\"[%d/%d] mean_loss : %0.2f\" %(epoch, EPOCH, np.mean(losses)))\n",
    "            losses = []"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "accuracy = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "97.61904761904762\n"
     ]
    }
   ],
   "source": [
    "for test in test_data:\n",
    "    pred = model(test[0]).max(1)[1]\n",
    "    pred = pred.data.tolist()[0]\n",
    "    target = test[1].data.tolist()[0][0]\n",
    "    if pred == target:\n",
    "        accuracy += 1\n",
    "\n",
    "print(accuracy/len(test_data) * 100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Further topics "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* <a href=\"https://arxiv.org/pdf/1508.06615.pdf\">Character-Aware Neural Language Models</a>\n",
    "* <a href=\"https://papers.nips.cc/paper/5782-character-level-convolutional-networks-for-text-classification.pdf\">Character level CNN for text classification</a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Suggested Reading"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* https://blog.statsbot.co/text-classifier-algorithms-in-machine-learning-acc115293278\n",
    "* <a href=\"https://arxiv.org/pdf/1607.01759\">Bag of Tricks for Efficient Text Classification</a>\n",
    "* <a href=\"https://arxiv.org/pdf/1708.02657\">Which Encoding is the Best for Text Classification in Chinese, English, Japanese and Korean?</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
